﻿Imports System.Threading
Imports System.Xml
Imports System.Xml.XPath
Imports System.Net
Imports System.IO


Public MustInherit Class base

    Protected Friend Const remaxUrl As String = "http://www.remax.com/realestatehomesforsale/"
    Protected Friend Const xmlns As String = "http://www.sitemaps.org/schemas/sitemap/0.9"
    Protected Friend isDelta As Boolean = False
    Protected Friend strXMLFolder As String
    Protected Friend strXMLFileName As String
    Protected Friend solrUrl As String
    Protected Friend solrID As String
    Protected Friend solrSelect1 As String
    Protected Friend solrSelect2 As String
    Protected Friend solrFields As String
    Protected Friend intTest As Integer = 0

    Protected Friend NodeAddress As Integer
    Protected Friend NodeCity As Integer
    Protected Friend NodeState As Integer
    Protected Friend NodeZip As Integer
    Protected Friend NodeHMID As Integer
    Protected Friend NodeMod As Integer

    Private sw As StreamWriter = Nothing
    Private stopLoop As Boolean = True
    Private increment As Integer
    Private minValue As Long = 0
    Private maxValue As Long = 0
    Private numFound As Long = 0
    Private threads As New List(Of Thread)
    Private docXmlExcess As XmlDocument = Nothing
    Private myLog As log


    Public MustOverride Sub StartProcess(ByVal inc As Integer, ByVal test As Integer, ByVal restart As Boolean)

#Region "   xml DOM process   "

    ' Return an XML doc
    Function GetXML(ByVal Page As String, Optional ByVal attempt As Integer = 0) As XmlDocument
        Dim responsedoc As XmlDocument = New XmlDocument()
        Dim rq As HttpWebRequest
        Dim resp As HttpWebResponse
        Dim reader As XmlTextReader
        Try
            rq = WebRequest.Create(Page)
            rq.Timeout = 120000
            resp = rq.GetResponse()
            reader = New XmlTextReader(resp.GetResponseStream())
            responsedoc.Load(reader)
            Return responsedoc
        Catch ex As Exception
            Console.ForegroundColor = ConsoleColor.Red
            Console.WriteLine(ex.Message)
            attempt += 1
            If attempt < 10 Then
                Thread.Sleep(1000)
                Return GetXML(Page, attempt)
            Else
                Return Nothing
            End If
        End Try
    End Function

    Function NewXmlSiteMap() As XmlDocument
        Dim doc As XmlDocument = New XmlDocument()
        'Create an XML declaration.  
        Dim xmldecl As XmlDeclaration = doc.CreateXmlDeclaration("1.0", "UTF-8", Nothing)
        Dim root As XmlElement = doc.DocumentElement
        doc.InsertBefore(xmldecl, root)
        Dim urlset As XmlElement = doc.AppendChild(doc.CreateElement("urlset"))
        urlset.SetAttribute("xmlns", xmlns)
        Return doc
    End Function

    'Process the given lists and create xml file
    Sub processXMLNodes(ByVal elem_list As List(Of XmlNodeList), ByVal Filecount As Long)
        Dim strAddress As String
        Dim strCity As String
        Dim strState As String
        Dim strZip As String
        Dim strHMID As String
        Dim strMod As String
        Dim intPos As Integer
        Dim dCount As Integer = 0

        Dim doc As XmlDocument = NewXmlSiteMap()
        Dim urlset As XmlElement = doc.GetElementsByTagName("urlset")(0)

        If docXmlExcess Is Nothing Then
            docXmlExcess = NewXmlSiteMap()
        End If
        Dim urlsetEx As XmlElement = docXmlExcess.GetElementsByTagName("urlset")(0)

        Dim url_e As XmlElement
        For I = 0 To elem_list.Count - 1
            For j = 0 To elem_list(I).Count - 1
                If elem_list(I)(j).ChildNodes().Count = 6 Then
                    strAddress = Replace(elem_list(I)(j).ChildNodes(NodeAddress).InnerText, " ", "-")
                    strCity = Replace(elem_list(I)(j).ChildNodes(NodeCity).InnerText, " ", "-")
                    strState = elem_list(I)(j).ChildNodes(NodeState).InnerText
                    strZip = elem_list(I)(j).ChildNodes(NodeZip).InnerText
                    strHMID = elem_list(I)(j).ChildNodes(NodeHMID).InnerText
                    strMod = elem_list(I)(j).ChildNodes(NodeMod).InnerText
                    intPos = InStr(strMod, "T")
                    If intPos > 0 Then
                        strMod = Left(strMod, intPos - 1)
                    End If
                    strAddress = remaxUrl & utils.CleanURLString(strAddress & "-" & strCity & "-" & strState & "-" & strZip) & "-" & "gid" & strHMID & ".html"

                    If dCount >= increment Then
                        url_e = urlsetEx.AppendChild(docXmlExcess.CreateElement("url"))
                        url_e.AppendChild(docXmlExcess.CreateElement("loc")).InnerText = strAddress
                        url_e.AppendChild(docXmlExcess.CreateElement("lastmod")).InnerText = strMod
                        If isDelta Then
                            url_e.AppendChild(docXmlExcess.CreateElement("changefreq")).InnerText = "daily"
                        End If
                    Else
                        url_e = urlset.AppendChild(doc.CreateElement("url"))
                        url_e.AppendChild(doc.CreateElement("loc")).InnerText = strAddress
                        url_e.AppendChild(doc.CreateElement("lastmod")).InnerText = strMod
                        If isDelta Then
                            url_e.AppendChild(doc.CreateElement("changefreq")).InnerText = "daily"
                        End If
                        dCount += 1
                    End If
                End If
            Next
        Next

        'Close the XML file
        If doc.GetElementsByTagName("url").Count > 0 Then
            Dim strDelta As String = ""
            If isDelta Then
                strDelta = "_" & DatePart("d", Now) & "_"
            End If
            doc.Save(strXMLFolder & "\" & strXMLFileName & strDelta & Filecount & ".xml")
        End If
    End Sub

    'Create the XML files
    Function createXML(ByVal Filecount As Long) As Long
        Dim respDoc As XmlDocument
        Dim elements As XmlNodeList
        Dim elem_list As New List(Of XmlNodeList)
        Dim docCount As Long = 0

        Dim smallInc As Integer = CInt(increment / 5)
        If isDelta Then smallInc = increment * 100
        Dim i As Integer = 0
        Do
            stopLoop = False
            Dim solrSelect As String = solrSelect1
            Console.ForegroundColor = i + 2
            i += 1
            If i > 5 Then i = 0
            solrSelect = solrSelect.Replace("@MIN@", minValue)
            If numFound <= increment Then
                solrSelect = solrSelect.Replace("@MAX@", maxValue)
                smallInc = maxValue - minValue
            Else
                solrSelect = solrSelect.Replace("@MAX@", minValue + smallInc)
            End If
            minValue += smallInc
            If minValue > maxValue Then minValue = maxValue
            Console.Write(solrSelect)
            myLog.Append(Filecount & " = " & solrSelect)

            respDoc = GetXML(solrUrl & solrSelect & solrSelect2 & solrFields & "&rows=" & smallInc)
            If respDoc Is Nothing Then
                Console.ForegroundColor = ConsoleColor.Red
                Console.WriteLine("")
                Console.WriteLine(" ERROR RETRIEVING SOLR DATA:")
                Console.WriteLine(solrUrl)
                Console.WriteLine("")
            Else
                elements = respDoc.GetElementsByTagName("doc")
                docCount += elements.Count
                numFound -= elements.Count
                If elements.Count = 0 Then
                    stopLoop = (minValue >= maxValue)
                    If Not stopLoop Then
                        minValue = GetID(CStr(minValue), CStr(maxValue), "asc")
                    End If
                Else
                    If elements.Count < 100 Then smallInc += increment
                    elem_list.Add(elements)
                End If
                Console.WriteLine("    " & CStr(docCount).PadLeft(10, " "))
            End If
        Loop Until (stopLoop Or docCount >= increment)

        Dim processXMLNod = New Thread(Sub() processXMLNodes(elem_list, Filecount))
        processXMLNod.Start()
        threads.Add(processXMLNod)
        Return docCount
    End Function

    'Get the ID from solr
    Function GetID(ByVal dMin As String, ByVal dMax As String, ByVal sort As String) As Long
        Dim myID As Long = 0
        Dim elements As XmlNodeList
        Dim respDoc As XmlDocument
        Dim solrSelect As String = solrSelect1
        solrSelect = solrSelect.Replace("@MIN@", dMin)
        solrSelect = solrSelect.Replace("@MAX@", dMax)
        Try
            respDoc = GetXML(solrUrl & solrSelect & solrSelect2 & "&rows=1&fl=" & solrID & "&sort=" & solrID & "+" & sort)
            elements = respDoc.GetElementsByTagName("doc")
            myID = CLng(elements(0).InnerText)
            If dMin = "*" Or dMax = "*" Then
                numFound = CLng(respDoc("response")("result").Attributes("numFound").InnerText)
            End If
        Catch ex As Exception
            Console.ForegroundColor = ConsoleColor.Red
            Console.WriteLine(ex.Message)
        End Try
        Return myID
    End Function

    Function doXMLExcess(ByVal Filecount As Long, Optional isFinal As Boolean = False) As Integer
        Dim intReponse As Integer = 0
        Dim elements As XmlNodeList
        If Not docXmlExcess Is Nothing Then
            Dim urlCount As Integer = docXmlExcess.GetElementsByTagName("url").Count
            If isFinal Or urlCount >= increment Then
                Dim strDelta As String = ""
                If isDelta Then
                    strDelta = "_" & DatePart("d", Now) & "_"
                End If

                For Each t As Thread In threads
                    t.Join()
                Next
                threads.Clear()                

                elements = docXmlExcess.GetElementsByTagName("url")
                Dim I As Integer
                Dim url_e As XmlElement
                Dim docXml As XmlDocument
                Dim urlset As XmlElement
                Do
                    docXml = NewXmlSiteMap()
                    urlset = docXml.GetElementsByTagName("urlset")(0)
                    For J As Integer = 0 To increment - 1
                        url_e = urlset.AppendChild(docXml.CreateElement("url"))
                        url_e.AppendChild(docXml.CreateElement("loc")).InnerText = elements(I).ChildNodes(0).InnerText
                        url_e.AppendChild(docXml.CreateElement("lastmod")).InnerText = elements(I).ChildNodes(1).InnerText
                        If isDelta Then
                            url_e.AppendChild(docXml.CreateElement("changefreq")).InnerText = "daily"
                        End If
                        I += 1
                        If I >= urlCount Then
                            Exit For
                        End If
                    Next
                    docXml.Save(strXMLFolder & "\" & strXMLFileName & strDelta & Filecount + intReponse & ".xml")
                    intReponse += 1
                Loop While ((urlCount - I) >= increment)

                docXmlExcess = Nothing
                If I < urlCount Then
                    docXmlExcess = NewXmlSiteMap()
                    urlset = docXml.GetElementsByTagName("urlset")(0)
                    For J As Integer = I To urlCount - 1
                        url_e = urlset.AppendChild(docXml.CreateElement("url"))
                        url_e.AppendChild(docXml.CreateElement("loc")).InnerText = elements(I).ChildNodes(0).InnerText
                        url_e.AppendChild(docXml.CreateElement("lastmod")).InnerText = elements(I).ChildNodes(1).InnerText
                        If isDelta Then
                            url_e.AppendChild(docXml.CreateElement("changefreq")).InnerText = "daily"
                        End If
                        I += 1
                    Next
                End If

                If isFinal And (Not docXmlExcess Is Nothing) Then
                    docXmlExcess.Save(strXMLFolder & "\" & strXMLFileName & strDelta & Filecount + intReponse & ".xml")
                    intReponse += 1
                End If
            End If
        End If
        Return intReponse
    End Function

    'Here is where it all starts
    Sub doXMlProcess(ByVal inc As Integer, ByVal restart As Boolean)
        Dim docCount As Long = 0
        Dim Filecount As Long = 0
        increment = inc

        Console.ForegroundColor = ConsoleColor.Green
        Console.WriteLine("")
        If restart Then
            Filecount = utils.getLatestFile(strXMLFolder, strXMLFileName)
            minValue = 900001982748
        Else
            Call utils.CheckFolderStatus(strXMLFolder, isDelta)
            minValue = GetID("*", "*", "asc")
        End If
        myLog = New log(strXMLFolder)

        Console.WriteLine("  Min " & solrID & ": " & CStr(minValue))
        maxValue = GetID(CStr(minValue), "*", "desc")
        Console.WriteLine("  Max " & solrID & ": " & CStr(maxValue))
        Console.WriteLine("")
        Console.WriteLine("  NumFound: " & CStr(numFound))
        Console.WriteLine("")
        Thread.Sleep(1000)

        Dim docXML As Long
        Do
            docXML = createXML(Filecount)
            If docXML > 0 Then
                docCount += docXML
                Filecount += 1
                Console.ForegroundColor = ConsoleColor.Green
                Console.WriteLine("")
                Console.WriteLine(numFound)
                Console.WriteLine("")
            End If            

            'for testing create only a few files
            If intTest > 2 Then
                stopLoop = (Filecount > intTest)
            End If
            'Process the Excess every 10 files
            If (Filecount Mod 10 = 0) Then
                Filecount += doXMLExcess(Filecount)
            End If
        Loop Until (stopLoop)

        Call doXMLExcess(Filecount, True)
        Call utils.WriteSiteMapIndex(strXMLFolder, strXMLFileName)
        myLog.Close()
    End Sub

#End Region


End Class
